import requests
import redis
from pyquery import PyQuery as pq

import sys
import time
import hashlib
import requests
# import grequests
from lxml import etree

_version = sys.version_info

is_python3 = (_version[0] == 3)

orderno = "ZF20194241515DHgKLd"
secret = "58b7019e8bf44c93a77e6b8ef2a506a4"

ip = "forward.xdaili.cn"
port = "80"

ip_port = ip + ":" + port

timestamp = str(int(time.time()))                # 计算时间戳
string = ""
string = "orderno=" + orderno + "," + "secret=" + secret + "," + "timestamp=" + timestamp

if is_python3:                          
    string = string.encode()

md5_string = hashlib.md5(string).hexdigest()                 # 计算sign
sign = md5_string.upper()                              # 转换成大写
print(sign)

auth = "sign=" + sign + "&" + "orderno=" + orderno + "&" + "timestamp=" + timestamp
print(auth)

proxy = {"http": "http://" + ip_port, "https": "https://" + ip_port}
print(proxy)

headers = {"Proxy-Authorization": auth}
res = requests.get("https://book.douban.com/tag/?view=type&icn=index-sorttags-all", headers=headers, proxies=proxy, verify=False,allow_redirects=False)

print("Status:%d"%res.status_code)
html = res.content.decode("utf-8")

# 数据解析
doc = pq(html)
alist = doc("table.tagCol tbody tr td a")

# 连接数据库
link = redis.StrictRedis(host="127.0.0.1",port=6379,db=0)

# 数据存入 Redis 数据库
# 将解析后的数据存入数据库
for item in alist.items():
	# 获得链接属性
	tag = item.attr.href
	# 存入数据库
	print(tag)
	link.lpush("book:tag_urls",tag)

print("一共存入了 %d 条数据..."%len(alist))

'''
/tag/经济学
/tag/管理
/tag/经济
/tag/商业
/tag/金融
/tag/投资
/tag/营销
/tag/理财
/tag/创业
/tag/股票
/tag/广告
/tag/企业史
/tag/策划
/tag/科普
/tag/互联网
/tag/编程
/tag/科学
/tag/交互设计
/tag/用户体验
/tag/算法
/tag/科技
/tag/web
/tag/交互
/tag/UE
/tag/通信
/tag/UCD
/tag/神经网络
/tag/程序
'''